package crawler;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import us.codecraft.webmagic.scheduler.PriorityScheduler;
import us.codecraft.webmagic.utils.HttpConstant;

public class Avmo implements PageProcessor{
	public static Map<String,List<String>> map=new HashMap();
	public static List<String> list=new ArrayList<>();
   //列表页 自己修改页码
	public static final String URL_LIST ="https://avmo\\.club/cn/released/page/10";
   //详情页
	public static final String IMAGE="https://avmo\\.club/cn/movie/\\w{4}";
   //搜索页
	public static final String SEARCH="https://btso\\.pw/search/\\w+";
    private Site site = new Site().me()
            .setRetryTimes(3).setTimeOut(5000000).setSleepTime(1000)
          //  addHeader("cookie", "__cfduid=d212f7d2d4ab12a7f621ff673b59bbd801508260445; _gat=1; AD_enterTime=1508260448; AD_exoc_j_L_728x90=0; AD_juic_j_L_728x90=1; AD_clic_j_POPUNDER=1; _ga=GA1.2.1091437298.1508260447; _gid=GA1.2.1562114948.1508260447; __test")
    		.addHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36")
    	//	.addHeader("referer:", "https://avmo.club/");
    		
    		;
    @Override
    public void process(Page page) {
    	page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());
    //	String tar="https://avmo.club/cn/movie/6c0k";
    	//page.addTargetRequest(tar);
        if (page.getUrl().regex(URL_LIST).match()) {
        	
        	page.addTargetRequests(	page.getHtml().xpath("//a[@class=\"movie-box\"]/@href").all());
        } else if(page.getUrl().regex(IMAGE).match()){
        	//map
        	List<String> maplist=new ArrayList<>();
        	
        	String key="";//番号
        	key=page.getHtml().xpath("//span[@style=\"color:#CC0000;\"]/text()").toString().trim();
        	String imagevalue="";//番号
        	imagevalue="image__"+page.getHtml().xpath("//div[@class=\"col-md-9 screencap\"]/a/img/@src").toString().trim();
        	maplist.add(imagevalue);//以image__地址 存进图片地址  
        	map.put(key, maplist);
        	List<String> list=page.getHtml().xpath("//div[@class=\"row ptb-10 text-center\"]/a/@href").all();
        	//list.add("https://btso.pw/search/dvdes");
        	for (String string : list) {
        		Request request=new Request();
        		request.setMethod(HttpConstant.Method.GET);
        		request.setUrl(string);
        		//request.
        		//request.addHeader("Host","btso.pw");
        		request.addHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0");
        		request.addHeader("Host", "btso.pw");
        		request.addHeader("Cache-Control", "no-cache");
        		request.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        		request.addHeader("Pragma", "no-cache");
        		request.addHeader("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
        		request.addHeader("Connection", "Keep-Alive");
        		page.addTargetRequest(request);
			}
        	
        } else if(page.getUrl().regex(SEARCH).match()){
        	List<String> list=page.getHtml().xpath("//div[@class=\"row\"]/a/@href").all();
        	//可能还有列表页暂时不写了
        	String key="";
        	key=page.getRequest().getUrl().split("search")[1].replace("/", "").trim();
        	List<String> maplist=map.get(key);
        	//list.add("https://btso.pw/search/dvdes");
        	if(list!=null){
        	for (String string : list) {
        		Request request=new Request();
        		request.setMethod(HttpConstant.Method.GET);
        		maplist.add("magnetList__"+string);
        		request.setUrl(string);
        		request.addHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0");
        		request.addHeader("Host", "btso.pw");
        		request.addHeader("Cache-Control", "no-cache");
        		request.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        		request.addHeader("Pragma", "no-cache");
        		request.addHeader("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
        		request.addHeader("Connection", "Keep-Alive");
        		page.addTargetRequest(request);
			}
        	}
        	//page.addTargetRequests(page.getHtml().xpath("//div[@class=\"row\"]/a/@href").all());
        }
        //磁链地址
        else {
        	//set.add(page.getHtml().xpath("//div[@class=\"container\"]/div[@class=\"}tags-box hidden-xs\"]/textarea").toString());
        	//磁链
        	///div[@class=\"container\"]/div[@class=\"tags-box hidden-xs\"]
        	/*     	String key="";
        	String value="";
        	map.get
        	map.get("");*/
        	//map
        	//key value
        	//番号	list 
        	//			→	image__
        	//			→	manageList__
        	//			→	后续处理将每个magnetList__ 返回一个string 磁链存入list
        	//				得到magnet__
        	//最后处理
        	//map
        	//key value
        	//番号	list 
        	//			→	image__
        	//			→	manageList__
        	//			→	magnet__
        	//			→	home__ 存储地址
        	System.out.println(page.getHtml().xpath("//textarea/text()").toString());
        }
    }
    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) throws Exception {
    	//建议设置翻墙代理
    	
    	 Spider.create(new Avmo()).addUrl("https://avmo.club/cn/released/page/10").thread(1).run();
    	  
    	 //map处理 得到相应关系
    	 mapMange(map);
    	 
    	 //新建文件夹存储图片磁链
    	 
    	 
    	 
    	 
    }
	public static Map<String, List<String>> mapMange(Map<String, List<String>> map2) {
		 Iterator<Map.Entry<String, List<String>>> it = map2.entrySet().iterator();
		    while (it.hasNext()) {
		    Map.Entry<String, List<String>> entry = it.next();
		    System.out.println("key= " + entry.getKey() + " and value= " + entry.getValue());
		    List<String> maplist=entry.getValue();
		    for (int i = 0; i < maplist.size(); i++) {
				if(maplist.get(i).split("__")[0].equals("magnetList")){
					if(null!=doget(maplist.get(i).split("__")[1])){
						try {
							Thread.sleep(1000);
							String	magnet=doget(maplist.get(i).split("__")[1]);
							maplist.add(magnet);
						} catch (InterruptedException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						}
						
					}
				}
			}
		    map2.put(entry.getKey(), maplist);
		   }
		    //测试是否添加
		 /*   Iterator<Map.Entry<String, List<String>>> it1 = map2.entrySet().iterator();
		    while (it1.hasNext()) {
			    Map.Entry<String, List<String>> entry = it1.next();
			    System.out.println("key= " + entry.getKey() + " and value= " + entry.getValue());
			    List<String> maplist=entry.getValue();
			   // List<String> list2=new ArrayList<>();
			    for (int i = 0; i < maplist.size(); i++) {
				System.out.println(maplist.get(i));
				}
		    }*/
		return map2;
	}
	
	private static String doget(String string) {
		HttpClient httpClient = new DefaultHttpClient();
		String url=string;
		String url2="";
	     // 获取当前客户端对象
	     // 通过请求对象获取响应对象
	     try {
	    
	    	 HttpGet httpGet=new HttpGet(url);
			 httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" );
			 httpGet.setHeader("Host", "btso.pw");
			 httpGet.setHeader("Cache-Control", "no-cache");
			 httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			 httpGet.setHeader("Pragma", "no-cache");
			 httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
			 httpGet.setHeader("Connection", "Keep-Alive");
			HttpResponse response = httpClient.execute(httpGet);
		String	 result= EntityUtils.toString(response.getEntity(),"utf-8");
		Document document = Jsoup.parse(result);
		Element e=document.getElementsByTag("textarea").get(1);
		url2=e.html();
		} catch (IOException e) {
			return null;
		}
		return url;
	}
}
